import json

import scrapy
from scrapy import FormRequest
from scrapy_redis.spiders import RedisSpider
from scrapy_redis.utils import bytes_to_str, is_dict, TextColor


class ConsumerSpider(RedisSpider):
    name = "consumer"
    redis_key = "tasks"

    def make_request_from_data(self, data):
        formatted_data = bytes_to_str(data, self.redis_encoding)

        if is_dict(formatted_data):
            parameter = json.loads(formatted_data)
        else:
            self.logger.warning(
                f"{TextColor.WARNING}WARNING: String request is deprecated, please use JSON data format. "
                f"Detail information, please check https://github.com/rmax/scrapy-redis#features{TextColor.ENDC}"
            )
            return FormRequest(formatted_data, dont_filter=True)

        if parameter.get("url", None) is None:
            self.logger.warning(
                f"{TextColor.WARNING}The data from Redis has no url key in push data{TextColor.ENDC}"
            )
            return []

        url = parameter.pop("url")
        method = parameter.pop("method").upper() if "method" in parameter else "GET"
        metadata = parameter.pop("meta") if "meta" in parameter else {}
        post_data = parameter.pop("data") if "data" in parameter else {}
        return FormRequest(
            url, dont_filter=True, method=method, formdata=post_data, meta=metadata
        )

    def parse(self, response):
        print(response.json())
